{
 "cells": [
  {
   "cell_type": "code",
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2024-09-24T03:57:59.453393Z",
     "start_time": "2024-09-24T03:51:34.567610Z"
    }
   },
   "source": [
    "# 数据处理、数据评分相关库\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "# 画图\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from scipy import stats\n",
    "%matplotlib inline\n",
    "plt.rcParams['font.sans-serif'] = 'SimHei'  # 设置中文显示\n",
    "plt.rcParams['font.size'] = 14  # 设置字体大小\n",
    "matplotlib.rcParams['axes.unicode_minus'] = False  # 解决负号问题\n",
    "\n",
    "#忽略警号\n",
    "import warnings\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "# user_log = pd.read_csv('../data/data_format1/sample_user_log_format1.csv',index_col='Unnamed: 0')# 少量样本数据，跑通模型\n",
    "test = pd.read_csv('C:\\\\Users\\\\20937\\\\Desktop\\\\阿里比赛\\\\data_format1\\\\test_format1.csv')\n",
    "train = pd.read_csv('C:\\\\Users\\\\20937\\\\Desktop\\\\阿里比赛\\\\data_format1\\\\train_format1.csv')\n",
    "user_info = pd.read_csv('C:\\\\Users\\\\20937\\\\Desktop\\\\阿里比赛\\\\data_format1\\\\user_info_format1.csv')\n",
    "user_log = pd.read_csv('C:\\\\Users\\\\20937\\\\Desktop\\\\阿里比赛\\\\data_format1\\\\user_log_format1.csv')\n",
    "train['origin'] = 'train'\n",
    "test['origin'] = 'test'\n",
    "# 连接train、test表\n",
    "data = pd.concat([train, test], ignore_index=True, sort=False)\n",
    "# 连接user_info表\n",
    "data = pd.merge(user_info, data, on='user_id', how='inner')\n",
    "data.drop(['prob'], axis=1, inplace=True)\n",
    "data.info()\n",
    "train['label'].value_counts() / len(train)  # 样本不均衡，需要处理\n",
    "# 整理user_log表\n",
    "user_log.rename(columns={'seller_id': 'merchant_id'}, inplace=True)\n",
    "user_log.info()\n",
    "data['user_id'] = data['user_id'].astype('int32')\n",
    "data['age_range'].fillna(0, inplace=True)  # 0和NULL表示未知\n",
    "data['age_range'] = data['age_range'].astype('int8')\n",
    "data['gender'].fillna(2, inplace=True)  #2和NULL表示未知\n",
    "data['gender'] = data['gender'].astype('int8')\n",
    "data['merchant_id'] = data['merchant_id'].astype('int32')\n",
    "data['label'] = data['label'].astype('str')\n",
    "data.info()\n",
    "user_log['user_id'] = user_log['user_id'].astype('int32')\n",
    "user_log['item_id'] = user_log['item_id'].astype('int32')\n",
    "user_log['cat_id'] = user_log['cat_id'].astype('int32')\n",
    "user_log['merchant_id'] = user_log['merchant_id'].astype('int32')\n",
    "user_log['brand_id'].fillna(0, inplace=True)\n",
    "user_log['brand_id'] = user_log['brand_id'].astype('int32')\n",
    "# 添加一个临时年\n",
    "user_log['time_stamp'] = user_log['time_stamp'].astype('str').apply(lambda x: '2020' + x)\n",
    "user_log['time_stamp'] = pd.to_datetime(user_log['time_stamp'], format='%Y%m%d')\n",
    "user_log['action_type'] = user_log['action_type'].astype('int8')\n",
    "user_log.info()\n",
    "groups = user_log.groupby(['user_id'])\n",
    "# 统计交互行为数量\n",
    "temp = groups.size().reset_index().rename(columns={0: 'u1'})\n",
    "\n",
    "temp\n",
    "data = pd.merge(data, temp, on='user_id', how='left')\n",
    "# 统计'item_id','cat_id','merchant_id','brand_id' 不重复值个数\n",
    "# temp = groups['item_id', 'cat_id', 'merchant_id', 'brand_id'].nunique().reset_index().rename(columns={\n",
    "#     'item_id': 'u2', 'cat_id': 'u3', 'merchant_id': 'u4', 'brand_id': 'u5'})\n",
    "# \n",
    "# temp\n",
    "temp = groups[['item_id', 'cat_id', 'merchant_id', 'brand_id']].nunique().reset_index().rename(columns={\n",
    "    'item_id': 'u2', 'cat_id': 'u3', 'merchant_id': 'u4', 'brand_id': 'u5'})\n",
    "\n",
    "temp\n",
    "data = pd.merge(data, temp, on='user_id', how='left')\n",
    "# 统计时间间隔\n",
    "temp = groups['time_stamp'].agg([('buy_far_time', 'min'), ('buy_late_time', 'max')]).reset_index()\n",
    "temp['u6'] = (temp['buy_late_time'] - temp['buy_far_time']).dt.days\n",
    "temp\n",
    "data = pd.merge(data, temp[['user_id', 'u6']], on='user_id', how='left')\n",
    "# 统计操作类型为0，1，2，3的个数\n",
    "temp = groups['action_type'].value_counts().unstack().reset_index().rename(\n",
    "    columns={0: 'u7', 1: 'u8', 2: 'u9', 3: 'u10'})\n",
    "\n",
    "temp\n",
    "data = pd.merge(data, temp[['user_id', 'u7', 'u8', 'u9', 'u10']], on='user_id', how='left')\n",
    "groups = user_log.groupby(['merchant_id'])\n",
    "# 统计 商家被交互行为数量\n",
    "temp = groups.size().reset_index().rename(columns={0: 'm1'})\n",
    "\n",
    "temp\n",
    "data = pd.merge(data, temp, on='merchant_id', how='left')\n",
    "# 统计'user_id','item_id','cat_id','brand_id' 不重复值个数\n",
    "temp = groups[['user_id', 'item_id', 'cat_id', 'brand_id']].nunique().reset_index().rename(columns={\n",
    "    'user_id': 'm2', 'item_id': 'm3', 'cat_id': 'm4', 'brand_id': 'm5'})\n",
    "\n",
    "temp\n",
    "data = pd.merge(data, temp, on='merchant_id', how='left')\n",
    "# 统计商家被交互的action_type数量\n",
    "temp = groups['action_type'].value_counts().unstack().reset_index().rename(columns={0: 'm6', 1: 'm7', 2: 'm8', 3: 'm9'})\n",
    "\n",
    "temp\n",
    "data = pd.merge(data, temp, on='merchant_id', how='left')\n",
    "# 按照merchant_id统计随机负采样的个数\n",
    "temp = train[train['label'] == 0].groupby(['merchant_id']).size().reset_index().rename(columns={0: 'm10'})\n",
    "\n",
    "temp\n",
    "data = pd.merge(data, temp, on='merchant_id', how='left')\n",
    "groups = user_log.groupby(['user_id', 'merchant_id'])\n",
    "# 统计交互行为数量\n",
    "temp = groups.size().reset_index().rename(columns={0: 'um1'})\n",
    "\n",
    "temp\n",
    "data = pd.merge(data, temp, on=['user_id', 'merchant_id'], how='left')\n",
    "# 统计'item_id','cat_id','brand_id' 不重复值个数\n",
    "temp = groups[['item_id', 'cat_id', 'brand_id']].nunique().reset_index().rename(columns={\n",
    "    'item_id': 'um2', 'cat_id': 'um3', 'brand_id': 'um4'})\n",
    "\n",
    "temp\n",
    "data = pd.merge(data, temp, on=['user_id', 'merchant_id'], how='left')\n",
    "# 统计操作类型为0，1，2，3的个数\n",
    "temp = groups['action_type'].value_counts().unstack().reset_index().rename(\n",
    "    columns={0: 'um5', 1: 'um6', 2: 'um7', 3: 'um8'})\n",
    "\n",
    "temp\n",
    "data = pd.merge(data, temp, on=['user_id', 'merchant_id'], how='left')\n",
    "# 统计时间间隔\n",
    "temp = groups['time_stamp'].agg([('buy_far_time', 'min'), ('buy_late_time', 'max')]).reset_index()\n",
    "temp['um9'] = (temp['buy_late_time'] - temp['buy_far_time']).dt.days\n",
    "temp\n",
    "data = pd.merge(data, temp[['user_id', 'merchant_id', 'um9']], on=['user_id', 'merchant_id'], how='left')\n",
    "# 用户购买点击比\n",
    "data['r1'] = data['u9'] / data['u7']\n",
    "# 商家购买点击比\n",
    "data['r2'] = data['m8'] / data['m6']\n",
    "# 不同用户不同商家购买点击比\n",
    "data['r3'] = data['um7'] / data['um5']\n",
    "# 年龄\n",
    "temp = pd.get_dummies(data['age_range'], prefix='age')\n",
    "\n",
    "temp\n",
    "data = pd.concat([data, temp], axis=1)\n",
    "# 性别\n",
    "temp = pd.get_dummies(data['gender'], prefix='gender')\n",
    "\n",
    "temp\n",
    "data = pd.concat([data, temp], axis=1)\n",
    "data.drop(columns=['age_range', 'gender'], inplace=True)\n",
    "train = data[data['origin'] == 'train'].drop(['origin'], axis=1)\n",
    "test = data[data['origin'] == 'test'].drop(['label', 'origin'], axis=1)\n",
    "X, Y = train.drop(['label'], axis=1), train['label']\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.2)\n",
    "X_train\n",
    "temp\n",
    "train_x, valid_x, train_y, valid_y = train_test_split(X, Y, test_size=0.2)\n",
    "from lightgbm import LGBMClassifier\n",
    "from sklearn.metrics import accuracy_score, roc_auc_score\n",
    "import lightgbm as lgb\n",
    "\n",
    "model = LGBMClassifier(\n",
    "    num_leaves=10,\n",
    "    learning_rate=0.05,\n",
    "    n_estimators=1000,\n",
    "    subsample=0.8,\n",
    ")\n",
    "\n",
    "early_stopping_callback = lgb.early_stopping(stopping_rounds=30, first_metric_only=False, verbose=True)\n",
    "\n",
    "model.fit(\n",
    "    train_x, train_y,\n",
    "    eval_set=[(train_x, train_y), (valid_x, valid_y)],\n",
    "    eval_metric='auc',\n",
    "    callbacks=[early_stopping_callback]\n",
    ")\n",
    "\n",
    "print('accuracy：', accuracy_score(Y, model.predict(X)))\n",
    "print('roc_auc：', roc_auc_score(Y, model.predict_proba(X)[:, 1]))\n",
    "\n",
    "prob = model.predict_proba(test)[:, 1]\n",
    "\n",
    "submission = pd.DataFrame()\n",
    "submission[['user_id', 'merchant_id']] = test[['user_id', 'merchant_id']]\n",
    "submission['prob'] = prob\n",
    "submission.to_csv('C:\\\\Users\\\\20937\\\\Desktop\\\\阿里比赛\\\\data_format1\\\\prediction_results.csv', index=False)"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 522341 entries, 0 to 522340\n",
      "Data columns (total 6 columns):\n",
      " #   Column       Non-Null Count   Dtype  \n",
      "---  ------       --------------   -----  \n",
      " 0   user_id      522341 non-null  int64  \n",
      " 1   age_range    519763 non-null  float64\n",
      " 2   gender       514796 non-null  float64\n",
      " 3   merchant_id  522341 non-null  int64  \n",
      " 4   label        260864 non-null  float64\n",
      " 5   origin       522341 non-null  object \n",
      "dtypes: float64(3), int64(2), object(1)\n",
      "memory usage: 23.9+ MB\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 54925330 entries, 0 to 54925329\n",
      "Data columns (total 7 columns):\n",
      " #   Column       Dtype  \n",
      "---  ------       -----  \n",
      " 0   user_id      int64  \n",
      " 1   item_id      int64  \n",
      " 2   cat_id       int64  \n",
      " 3   merchant_id  int64  \n",
      " 4   brand_id     float64\n",
      " 5   time_stamp   int64  \n",
      " 6   action_type  int64  \n",
      "dtypes: float64(1), int64(6)\n",
      "memory usage: 2.9 GB\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 522341 entries, 0 to 522340\n",
      "Data columns (total 6 columns):\n",
      " #   Column       Non-Null Count   Dtype \n",
      "---  ------       --------------   ----- \n",
      " 0   user_id      522341 non-null  int32 \n",
      " 1   age_range    522341 non-null  int8  \n",
      " 2   gender       522341 non-null  int8  \n",
      " 3   merchant_id  522341 non-null  int32 \n",
      " 4   label        522341 non-null  object\n",
      " 5   origin       522341 non-null  object\n",
      "dtypes: int32(2), int8(2), object(2)\n",
      "memory usage: 13.0+ MB\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 54925330 entries, 0 to 54925329\n",
      "Data columns (total 7 columns):\n",
      " #   Column       Dtype         \n",
      "---  ------       -----         \n",
      " 0   user_id      int32         \n",
      " 1   item_id      int32         \n",
      " 2   cat_id       int32         \n",
      " 3   merchant_id  int32         \n",
      " 4   brand_id     int32         \n",
      " 5   time_stamp   datetime64[ns]\n",
      " 6   action_type  int8          \n",
      "dtypes: datetime64[ns](1), int32(5), int8(1)\n",
      "memory usage: 1.5 GB\n",
      "[LightGBM] [Info] Number of positive: 12729, number of negative: 195962\n",
      "[LightGBM] [Info] Auto-choosing row-wise multi-threading, the overhead of testing was 0.029474 seconds.\n",
      "You can set `force_row_wise=true` to remove the overhead.\n",
      "And if memory is not enough, you can set `force_col_wise=true`.\n",
      "[LightGBM] [Info] Total Bins 6180\n",
      "[LightGBM] [Info] Number of data points in the train set: 208691, number of used features: 45\n",
      "[LightGBM] [Info] [binary:BoostFromScore]: pavg=0.060994 -> initscore=-2.734038\n",
      "[LightGBM] [Info] Start training from score -2.734038\n",
      "Training until validation scores don't improve for 30 rounds\n",
      "Early stopping, best iteration is:\n",
      "[572]\ttraining's auc: 0.737995\ttraining's binary_logloss: 0.207007\tvalid_1's auc: 0.68838\tvalid_1's binary_logloss: 0.217961\n",
      "accuracy： 0.9390180323846908\n",
      "roc_auc： 0.7280277265085582\n"
     ]
    }
   ],
   "execution_count": 1
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
